{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from dynamic_sparse.common.browser import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and check data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = ['neurips_1_eval1', ]\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>lr_scheduler</th>\n",
       "      <th>model</th>\n",
       "      <th>momentum</th>\n",
       "      <th>network</th>\n",
       "      <th>num_classes</th>\n",
       "      <th>on_perc</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th>test_noise</th>\n",
       "      <th>weight_decay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_pruning_early_stop=0</td>\n",
       "      <td>0.998533</td>\n",
       "      <td>73</td>\n",
       "      <td>0.910383</td>\n",
       "      <td>0</td>\n",
       "      <td>0.997575</td>\n",
       "      <td>0.998417</td>\n",
       "      <td>0.9807</td>\n",
       "      <td>86</td>\n",
       "      <td>0.9454</td>\n",
       "      <td>...</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>SET</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLP</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_pruning_early_stop=1</td>\n",
       "      <td>0.999000</td>\n",
       "      <td>96</td>\n",
       "      <td>0.909583</td>\n",
       "      <td>0</td>\n",
       "      <td>0.997783</td>\n",
       "      <td>0.998367</td>\n",
       "      <td>0.9814</td>\n",
       "      <td>69</td>\n",
       "      <td>0.9513</td>\n",
       "      <td>...</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>SET</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLP</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>SGD</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_pruning_early_stop=2</td>\n",
       "      <td>0.998033</td>\n",
       "      <td>99</td>\n",
       "      <td>0.907067</td>\n",
       "      <td>0</td>\n",
       "      <td>0.997092</td>\n",
       "      <td>0.998033</td>\n",
       "      <td>0.9795</td>\n",
       "      <td>35</td>\n",
       "      <td>0.9560</td>\n",
       "      <td>...</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>SET</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLP</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>SGD</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_pruning_early_stop=3</td>\n",
       "      <td>0.998467</td>\n",
       "      <td>74</td>\n",
       "      <td>0.911750</td>\n",
       "      <td>0</td>\n",
       "      <td>0.996508</td>\n",
       "      <td>0.996817</td>\n",
       "      <td>0.9828</td>\n",
       "      <td>35</td>\n",
       "      <td>0.9480</td>\n",
       "      <td>...</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>SET</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLP</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>SGD</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_pruning_early_stop=0</td>\n",
       "      <td>0.998350</td>\n",
       "      <td>78</td>\n",
       "      <td>0.911233</td>\n",
       "      <td>0</td>\n",
       "      <td>0.997158</td>\n",
       "      <td>0.997400</td>\n",
       "      <td>0.9816</td>\n",
       "      <td>37</td>\n",
       "      <td>0.9531</td>\n",
       "      <td>...</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>SET</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLP</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Experiment Name  train_acc_max  train_acc_max_epoch  train_acc_min  \\\n",
       "0  0_pruning_early_stop=0       0.998533                   73       0.910383   \n",
       "1  1_pruning_early_stop=1       0.999000                   96       0.909583   \n",
       "2  2_pruning_early_stop=2       0.998033                   99       0.907067   \n",
       "3  3_pruning_early_stop=3       0.998467                   74       0.911750   \n",
       "4  4_pruning_early_stop=0       0.998350                   78       0.911233   \n",
       "\n",
       "   train_acc_min_epoch  train_acc_median  train_acc_last  val_acc_max  \\\n",
       "0                    0          0.997575        0.998417       0.9807   \n",
       "1                    0          0.997783        0.998367       0.9814   \n",
       "2                    0          0.997092        0.998033       0.9795   \n",
       "3                    0          0.996508        0.996817       0.9828   \n",
       "4                    0          0.997158        0.997400       0.9816   \n",
       "\n",
       "   val_acc_max_epoch  val_acc_min  ...  lr_scheduler  model  momentum  \\\n",
       "0                 86       0.9454  ...   MultiStepLR    SET       0.9   \n",
       "1                 69       0.9513  ...   MultiStepLR    SET       0.9   \n",
       "2                 35       0.9560  ...   MultiStepLR    SET       0.9   \n",
       "3                 35       0.9480  ...   MultiStepLR    SET       0.9   \n",
       "4                 37       0.9531  ...   MultiStepLR    SET       0.9   \n",
       "\n",
       "   network num_classes  on_perc  optim_alg  pruning_early_stop test_noise  \\\n",
       "0      MLP          10      0.1        SGD                   0      False   \n",
       "1      MLP          10      0.1        SGD                   1      False   \n",
       "2      MLP          10      0.1        SGD                   2      False   \n",
       "3      MLP          10      0.1        SGD                   3      False   \n",
       "4      MLP          10      0.1        SGD                   0      False   \n",
       "\n",
       "  weight_decay  \n",
       "0       0.0001  \n",
       "1       0.0001  \n",
       "2       0.0001  \n",
       "3       0.0001  \n",
       "4       0.0001  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'epochs',\n",
       "       'experiment_file_name', 'trial_time', 'mean_epoch_time', 'batch_norm',\n",
       "       'data_dir', 'dataset_name', 'debug_sparse', 'debug_weights', 'device',\n",
       "       'hebbian_grow', 'hebbian_prune_perc', 'hidden_sizes', 'input_size',\n",
       "       'kwinners', 'learning_rate', 'lr_gamma', 'lr_milestones',\n",
       "       'lr_scheduler', 'model', 'momentum', 'network', 'num_classes',\n",
       "       'on_perc', 'optim_alg', 'pruning_early_stop', 'test_noise',\n",
       "       'weight_decay'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 41)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name                                    1_pruning_early_stop=1\n",
       "train_acc_max                                                       0.999\n",
       "train_acc_max_epoch                                                    96\n",
       "train_acc_min                                                    0.909583\n",
       "train_acc_min_epoch                                                     0\n",
       "train_acc_median                                                 0.997783\n",
       "train_acc_last                                                   0.998367\n",
       "val_acc_max                                                        0.9814\n",
       "val_acc_max_epoch                                                      69\n",
       "val_acc_min                                                        0.9513\n",
       "val_acc_min_epoch                                                       0\n",
       "val_acc_median                                                     0.9803\n",
       "val_acc_last                                                       0.9805\n",
       "epochs                                                                100\n",
       "experiment_file_name    /Users/lsouza/nta/results/neurips_1_eval1/expe...\n",
       "trial_time                                                        20.0659\n",
       "mean_epoch_time                                                  0.200659\n",
       "batch_norm                                                           True\n",
       "data_dir                                        /home/ubuntu/nta/datasets\n",
       "dataset_name                                                        MNIST\n",
       "debug_sparse                                                         True\n",
       "debug_weights                                                        True\n",
       "device                                                               cuda\n",
       "hebbian_grow                                                        False\n",
       "hebbian_prune_perc                                                    0.3\n",
       "hidden_sizes                                                          100\n",
       "input_size                                                            784\n",
       "kwinners                                                            False\n",
       "learning_rate                                                         0.1\n",
       "lr_gamma                                                              0.1\n",
       "lr_milestones                                                          60\n",
       "lr_scheduler                                                  MultiStepLR\n",
       "model                                                                 SET\n",
       "momentum                                                              0.9\n",
       "network                                                               MLP\n",
       "num_classes                                                            10\n",
       "on_perc                                                               0.1\n",
       "optim_alg                                                             SGD\n",
       "pruning_early_stop                                                      1\n",
       "test_noise                                                          False\n",
       "weight_decay                                                       0.0001\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "SET    10\n",
       "Name: model, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('model')['model'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment Details"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "base_exp_config = dict(\n",
    "    device=\"cuda\",\n",
    "    # dataset related\n",
    "    dataset_name=\"MNIST\",\n",
    "    data_dir=os.path.expanduser(\"~/nta/datasets\"),\n",
    "    input_size=784,\n",
    "    num_classes=10,\n",
    "    # network related\n",
    "    network=\"MLP\", # \"MLPHeb\",\n",
    "    hidden_sizes=[100, 100, 100],\n",
    "    batch_norm=True,\n",
    "    kwinners=False,\n",
    "    # model related\n",
    "    model=\"SET\", #\"DSNNMixedHeb\",\n",
    "    on_perc=0.1,\n",
    "    optim_alg=\"SGD\",\n",
    "    momentum=0.9,\n",
    "    weight_decay=1e-4,    \n",
    "    learning_rate=0.1,\n",
    "    lr_scheduler=\"MultiStepLR\",\n",
    "    lr_milestones=[30,60,90],\n",
    "    lr_gamma=0.1,\n",
    "    # sparse related\n",
    "    hebbian_prune_perc=0.3,\n",
    "    pruning_early_stop=1, #tune.grid_search([None, 1, 2, 3]),\n",
    "    hebbian_grow=False,\n",
    "    # additional validation\n",
    "    test_noise=False,\n",
    "    # debugging\n",
    "    debug_weights=True,\n",
    "    debug_sparse=True,\n",
    "    stop={\"training_iteration\": 100},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Did any  trials failed?\n",
    "df[df[\"epochs\"]<30][\"epochs\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 41)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing failed trials\n",
    "df_origin = df.copy()\n",
    "df = df_origin[df_origin[\"epochs\"]>=30]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8    80\n",
       "9    63\n",
       "Name: epochs, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# which ones failed?\n",
    "# failed, or still ongoing?\n",
    "df_origin['failed'] = df_origin[\"epochs\"]<100\n",
    "df_origin[df_origin['failed']]['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.3f} ± {:.3f}\".format(s.mean(), s.std())\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Did Hebbian perform better than SET?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>SET</th>\n",
       "      <td>56</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.001</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      val_acc_max_epoch val_acc_max                      model\n",
       "             round_mean         min    max   mean    std count\n",
       "model                                                         \n",
       "SET                  56        0.98  0.983  0.981  0.001    10"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "high_sparsity = (df['on_perc']==0.05)\n",
    "avg_sparsity = (df['on_perc']==0.1)\n",
    "low_sparsity = (df['on_perc']==0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kwinners</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>21</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.009</td>\n",
       "      <td>19</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.011</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>22</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.008</td>\n",
       "      <td>20</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.010</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         val_acc_max_epoch val_acc_max                       \\\n",
       "                round_mean         min    max   mean    std   \n",
       "kwinners                                                      \n",
       "False                   21       0.932  0.978  0.963  0.009   \n",
       "True                    22       0.938  0.976  0.964  0.008   \n",
       "\n",
       "         noise_acc_max_epoch noise_acc_max                      model  \n",
       "                  round_mean           min    max   mean    std count  \n",
       "kwinners                                                               \n",
       "False                     19         0.908  0.963  0.948  0.011    69  \n",
       "True                      20         0.911  0.963  0.949  0.010    68  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['kwinners'], low_sparsity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kwinners</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>24</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.128</td>\n",
       "      <td>22</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.872</td>\n",
       "      <td>0.121</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>23</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.882</td>\n",
       "      <td>0.166</td>\n",
       "      <td>22</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.157</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         val_acc_max_epoch val_acc_max                       \\\n",
       "                round_mean         min    max   mean    std   \n",
       "kwinners                                                      \n",
       "False                   24       0.398  0.969  0.904  0.128   \n",
       "True                    23       0.295  0.967  0.882  0.166   \n",
       "\n",
       "         noise_acc_max_epoch noise_acc_max                      model  \n",
       "                  round_mean           min    max   mean    std count  \n",
       "kwinners                                                               \n",
       "False                     22         0.392  0.941  0.872  0.121    37  \n",
       "True                      22         0.292  0.937  0.851  0.157    39  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['kwinners'], high_sparsity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* No evidence of significant difference. In networks with high sparsity, the impact of kWinners is worst, which is expected since kWinners (at 30%) will make the activations more sparse than ReLU (which is 50% sparse on average)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### What is the optimal level of weight sparsity?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>on_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>24</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.893</td>\n",
       "      <td>0.148</td>\n",
       "      <td>22</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.861</td>\n",
       "      <td>0.140</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.10</th>\n",
       "      <td>22</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.020</td>\n",
       "      <td>21</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.929</td>\n",
       "      <td>0.024</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.20</th>\n",
       "      <td>22</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.009</td>\n",
       "      <td>19</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.010</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        val_acc_max_epoch val_acc_max                       \\\n",
       "               round_mean         min    max   mean    std   \n",
       "on_perc                                                      \n",
       "0.05                   24       0.295  0.969  0.893  0.148   \n",
       "0.10                   22       0.888  0.974  0.952  0.020   \n",
       "0.20                   22       0.932  0.978  0.963  0.009   \n",
       "\n",
       "        noise_acc_max_epoch noise_acc_max                      model  \n",
       "                 round_mean           min    max   mean    std count  \n",
       "on_perc                                                               \n",
       "0.05                     22         0.292  0.941  0.861  0.140    76  \n",
       "0.10                     21         0.851  0.954  0.929  0.024   130  \n",
       "0.20                     19         0.908  0.963  0.948  0.010   137  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['on_perc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Sparsity at 80 and 90% levels seem more or less equivalent, difference is 1 point in accuracy. The jump from 90 to 95% shows a drastic increase in acc, of 6 points."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Hebbian grow helps learning?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_grow</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>22</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.956</td>\n",
       "      <td>0.017</td>\n",
       "      <td>21</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.022</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>22</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.106</td>\n",
       "      <td>20</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.105</td>\n",
       "      <td>167</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             val_acc_max_epoch val_acc_max                       \\\n",
       "                    round_mean         min    max   mean    std   \n",
       "hebbian_grow                                                      \n",
       "False                       22       0.888  0.978  0.956  0.017   \n",
       "True                        22       0.295  0.972  0.930  0.106   \n",
       "\n",
       "             noise_acc_max_epoch noise_acc_max                      model  \n",
       "                      round_mean           min    max   mean    std count  \n",
       "hebbian_grow                                                               \n",
       "False                         21         0.851  0.963  0.932  0.022   176  \n",
       "True                          20         0.292  0.960  0.911  0.105   167  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_grow'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_grow</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>20</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.011</td>\n",
       "      <td>18</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.013</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>23</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.005</td>\n",
       "      <td>21</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.006</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             val_acc_max_epoch val_acc_max                       \\\n",
       "                    round_mean         min    max   mean    std   \n",
       "hebbian_grow                                                      \n",
       "False                       20       0.932  0.978  0.962  0.011   \n",
       "True                        23       0.938  0.972  0.965  0.005   \n",
       "\n",
       "             noise_acc_max_epoch noise_acc_max                      model  \n",
       "                      round_mean           min    max   mean    std count  \n",
       "hebbian_grow                                                               \n",
       "False                         18         0.908  0.963  0.945  0.013    65  \n",
       "True                          21         0.914  0.960  0.951  0.006    72  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_grow'], low_sparsity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_grow</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>25</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.016</td>\n",
       "      <td>24</td>\n",
       "      <td>0.853</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.916</td>\n",
       "      <td>0.019</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>22</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.793</td>\n",
       "      <td>0.210</td>\n",
       "      <td>19</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.769</td>\n",
       "      <td>0.200</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             val_acc_max_epoch val_acc_max                       \\\n",
       "                    round_mean         min    max   mean    std   \n",
       "hebbian_grow                                                      \n",
       "False                       25       0.905  0.969  0.951  0.016   \n",
       "True                        22       0.295  0.954  0.793  0.210   \n",
       "\n",
       "             noise_acc_max_epoch noise_acc_max                      model  \n",
       "                      round_mean           min    max   mean    std count  \n",
       "hebbian_grow                                                               \n",
       "False                         24         0.853  0.941  0.916  0.019    48  \n",
       "True                          19         0.292  0.923  0.769  0.200    28  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_grow'], high_sparsity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* No strong evidence it helps in low sparsity case. In high sparsity (95%), seems very harmful"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Hebbian pruning helps learning?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>16</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.024</td>\n",
       "      <td>16</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.033</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>20</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.019</td>\n",
       "      <td>20</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.026</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>23</td>\n",
       "      <td>0.402</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.102</td>\n",
       "      <td>22</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.101</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>23</td>\n",
       "      <td>0.538</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.078</td>\n",
       "      <td>22</td>\n",
       "      <td>0.530</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.918</td>\n",
       "      <td>0.077</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>24</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.073</td>\n",
       "      <td>22</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.931</td>\n",
       "      <td>0.072</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>23</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.085</td>\n",
       "      <td>19</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.084</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                       \\\n",
       "                          round_mean         min    max   mean    std   \n",
       "hebbian_prune_perc                                                      \n",
       "0.0                               16       0.888  0.971  0.943  0.024   \n",
       "0.1                               20       0.900  0.969  0.942  0.019   \n",
       "0.2                               23       0.402  0.970  0.928  0.102   \n",
       "0.3                               23       0.538  0.968  0.939  0.078   \n",
       "0.4                               24       0.398  0.974  0.953  0.073   \n",
       "0.5                               23       0.295  0.978  0.953  0.085   \n",
       "\n",
       "                   noise_acc_max_epoch noise_acc_max                       \\\n",
       "                            round_mean           min    max   mean    std   \n",
       "hebbian_prune_perc                                                          \n",
       "0.0                                 16         0.851  0.959  0.923  0.033   \n",
       "0.1                                 20         0.868  0.957  0.922  0.026   \n",
       "0.2                                 22         0.396  0.960  0.907  0.101   \n",
       "0.3                                 22         0.530  0.952  0.918  0.077   \n",
       "0.4                                 22         0.392  0.960  0.931  0.072   \n",
       "0.5                                 19         0.292  0.963  0.930  0.084   \n",
       "\n",
       "                   model  \n",
       "                   count  \n",
       "hebbian_prune_perc        \n",
       "0.0                   31  \n",
       "0.1                   45  \n",
       "0.2                   64  \n",
       "0.3                   67  \n",
       "0.4                   67  \n",
       "0.5                   69  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_prune_perc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>18</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.956</td>\n",
       "      <td>0.011</td>\n",
       "      <td>18</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.016</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>22</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.955</td>\n",
       "      <td>0.008</td>\n",
       "      <td>22</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.011</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>22</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.006</td>\n",
       "      <td>21</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.946</td>\n",
       "      <td>0.009</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>23</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.002</td>\n",
       "      <td>21</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>23</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.003</td>\n",
       "      <td>20</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.003</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>21</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.003</td>\n",
       "      <td>15</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.004</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                       \\\n",
       "                          round_mean         min    max   mean    std   \n",
       "hebbian_prune_perc                                                      \n",
       "0.0                               18       0.932  0.971  0.956  0.011   \n",
       "0.1                               22       0.945  0.969  0.955  0.008   \n",
       "0.2                               22       0.938  0.970  0.961  0.006   \n",
       "0.3                               23       0.960  0.968  0.964  0.002   \n",
       "0.4                               23       0.964  0.974  0.969  0.003   \n",
       "0.5                               21       0.967  0.978  0.972  0.003   \n",
       "\n",
       "                   noise_acc_max_epoch noise_acc_max                       \\\n",
       "                            round_mean           min    max   mean    std   \n",
       "hebbian_prune_perc                                                          \n",
       "0.0                                 18         0.908  0.959  0.941  0.016   \n",
       "0.1                                 22         0.925  0.957  0.940  0.011   \n",
       "0.2                                 21         0.914  0.960  0.946  0.009   \n",
       "0.3                                 21         0.945  0.952  0.950  0.002   \n",
       "0.4                                 20         0.948  0.960  0.954  0.003   \n",
       "0.5                                 15         0.950  0.963  0.957  0.004   \n",
       "\n",
       "                   model  \n",
       "                   count  \n",
       "hebbian_prune_perc        \n",
       "0.0                   17  \n",
       "0.1                   24  \n",
       "0.2                   24  \n",
       "0.3                   24  \n",
       "0.4                   24  \n",
       "0.5                   24  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_prune_perc'], low_sparsity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>29</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29</td>\n",
       "      <td>0.879</td>\n",
       "      <td>0.879</td>\n",
       "      <td>0.879</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>25</td>\n",
       "      <td>0.402</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.847</td>\n",
       "      <td>0.185</td>\n",
       "      <td>24</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.815</td>\n",
       "      <td>0.174</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>21</td>\n",
       "      <td>0.538</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.882</td>\n",
       "      <td>0.131</td>\n",
       "      <td>21</td>\n",
       "      <td>0.530</td>\n",
       "      <td>0.926</td>\n",
       "      <td>0.852</td>\n",
       "      <td>0.124</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>25</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.918</td>\n",
       "      <td>0.133</td>\n",
       "      <td>23</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.886</td>\n",
       "      <td>0.126</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>24</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.149</td>\n",
       "      <td>21</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.882</td>\n",
       "      <td>0.142</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                       \\\n",
       "                          round_mean         min    max   mean    std   \n",
       "hebbian_prune_perc                                                      \n",
       "0.1                               29       0.900  0.900  0.900    NaN   \n",
       "0.2                               25       0.402  0.952  0.847  0.185   \n",
       "0.3                               21       0.538  0.952  0.882  0.131   \n",
       "0.4                               25       0.398  0.966  0.918  0.133   \n",
       "0.5                               24       0.295  0.969  0.914  0.149   \n",
       "\n",
       "                   noise_acc_max_epoch noise_acc_max                       \\\n",
       "                            round_mean           min    max   mean    std   \n",
       "hebbian_prune_perc                                                          \n",
       "0.1                                 29         0.879  0.879  0.879    NaN   \n",
       "0.2                                 24         0.396  0.920  0.815  0.174   \n",
       "0.3                                 21         0.530  0.926  0.852  0.124   \n",
       "0.4                                 23         0.392  0.937  0.886  0.126   \n",
       "0.5                                 21         0.292  0.941  0.882  0.142   \n",
       "\n",
       "                   model  \n",
       "                   count  \n",
       "hebbian_prune_perc        \n",
       "0.1                    1  \n",
       "0.2                   16  \n",
       "0.3                   19  \n",
       "0.4                   19  \n",
       "0.5                   21  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_prune_perc'], high_sparsity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* There is good evidence it helps. The trend is very clear in the low sparsity (80% sparse) cases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>15</td>\n",
       "      <td>0.895</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.032</td>\n",
       "      <td>15</td>\n",
       "      <td>0.853</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.047</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>18</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.022</td>\n",
       "      <td>17</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.028</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>23</td>\n",
       "      <td>0.402</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.167</td>\n",
       "      <td>22</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.885</td>\n",
       "      <td>0.163</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>21</td>\n",
       "      <td>0.561</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.906</td>\n",
       "      <td>0.124</td>\n",
       "      <td>21</td>\n",
       "      <td>0.549</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.886</td>\n",
       "      <td>0.122</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>24</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.010</td>\n",
       "      <td>22</td>\n",
       "      <td>0.903</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.934</td>\n",
       "      <td>0.017</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>22</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.009</td>\n",
       "      <td>16</td>\n",
       "      <td>0.901</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.017</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                       \\\n",
       "                          round_mean         min    max   mean    std   \n",
       "hebbian_prune_perc                                                      \n",
       "0.0                               15       0.895  0.971  0.939  0.032   \n",
       "0.1                               18       0.905  0.965  0.940  0.022   \n",
       "0.2                               23       0.402  0.970  0.905  0.167   \n",
       "0.3                               21       0.561  0.964  0.906  0.124   \n",
       "0.4                               24       0.938  0.968  0.960  0.010   \n",
       "0.5                               22       0.942  0.973  0.963  0.009   \n",
       "\n",
       "                   noise_acc_max_epoch noise_acc_max                       \\\n",
       "                            round_mean           min    max   mean    std   \n",
       "hebbian_prune_perc                                                          \n",
       "0.0                                 15         0.853  0.959  0.915  0.047   \n",
       "0.1                                 17         0.880  0.953  0.920  0.028   \n",
       "0.2                                 22         0.396  0.960  0.885  0.163   \n",
       "0.3                                 21         0.549  0.951  0.886  0.122   \n",
       "0.4                                 22         0.903  0.954  0.934  0.017   \n",
       "0.5                                 16         0.901  0.954  0.935  0.017   \n",
       "\n",
       "                   model  \n",
       "                   count  \n",
       "hebbian_prune_perc        \n",
       "0.0                    5  \n",
       "0.1                    8  \n",
       "0.2                   11  \n",
       "0.3                   12  \n",
       "0.4                   12  \n",
       "0.5                   12  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_magnitude = (df['weight_prune_perc'] == 0)\n",
    "agg(['hebbian_prune_perc'], no_magnitude)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>24</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.009</td>\n",
       "      <td>29</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.008</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>23</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.956</td>\n",
       "      <td>0.009</td>\n",
       "      <td>21</td>\n",
       "      <td>0.931</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.012</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>21</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.005</td>\n",
       "      <td>20</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.009</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>24</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.001</td>\n",
       "      <td>24</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.003</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>22</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.001</td>\n",
       "      <td>24</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.003</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>20</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.002</td>\n",
       "      <td>10</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.002</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                       \\\n",
       "                          round_mean         min    max   mean    std   \n",
       "hebbian_prune_perc                                                      \n",
       "0.0                               24       0.959  0.971  0.965  0.009   \n",
       "0.1                               23       0.947  0.965  0.956  0.009   \n",
       "0.2                               21       0.958  0.970  0.963  0.005   \n",
       "0.3                               24       0.963  0.964  0.964  0.001   \n",
       "0.4                               22       0.966  0.968  0.967  0.001   \n",
       "0.5                               20       0.968  0.973  0.971  0.002   \n",
       "\n",
       "                   noise_acc_max_epoch noise_acc_max                       \\\n",
       "                            round_mean           min    max   mean    std   \n",
       "hebbian_prune_perc                                                          \n",
       "0.0                                 29         0.947  0.959  0.953  0.008   \n",
       "0.1                                 21         0.931  0.953  0.942  0.012   \n",
       "0.2                                 20         0.941  0.960  0.948  0.009   \n",
       "0.3                                 24         0.945  0.951  0.948  0.003   \n",
       "0.4                                 24         0.948  0.954  0.950  0.003   \n",
       "0.5                                 10         0.950  0.954  0.952  0.002   \n",
       "\n",
       "                   model  \n",
       "                   count  \n",
       "hebbian_prune_perc        \n",
       "0.0                    2  \n",
       "0.1                    4  \n",
       "0.2                    4  \n",
       "0.3                    4  \n",
       "0.4                    4  \n",
       "0.5                    4  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_magnitude = (df['weight_prune_perc'] == 0)\n",
    "agg(['hebbian_prune_perc'], (no_magnitude & low_sparsity))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Results seem similar even when no magnitude pruning is involved, only hebbian pruning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Magnitude pruning helps learning?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>21</td>\n",
       "      <td>0.402</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.091</td>\n",
       "      <td>19</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.912</td>\n",
       "      <td>0.090</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>22</td>\n",
       "      <td>0.585</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.060</td>\n",
       "      <td>21</td>\n",
       "      <td>0.573</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.919</td>\n",
       "      <td>0.061</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>24</td>\n",
       "      <td>0.862</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.018</td>\n",
       "      <td>23</td>\n",
       "      <td>0.836</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.022</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>22</td>\n",
       "      <td>0.458</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.946</td>\n",
       "      <td>0.073</td>\n",
       "      <td>20</td>\n",
       "      <td>0.452</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.073</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>22</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.094</td>\n",
       "      <td>22</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.918</td>\n",
       "      <td>0.093</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>22</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.092</td>\n",
       "      <td>19</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.921</td>\n",
       "      <td>0.091</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                       \\\n",
       "                         round_mean         min    max   mean    std   \n",
       "weight_prune_perc                                                      \n",
       "0.0                              21       0.402  0.973  0.935  0.091   \n",
       "0.1                              22       0.585  0.976  0.942  0.060   \n",
       "0.2                              24       0.862  0.976  0.957  0.018   \n",
       "0.3                              22       0.458  0.976  0.946  0.073   \n",
       "0.4                              22       0.398  0.978  0.938  0.094   \n",
       "0.5                              22       0.295  0.976  0.943  0.092   \n",
       "\n",
       "                  noise_acc_max_epoch noise_acc_max                      model  \n",
       "                           round_mean           min    max   mean    std count  \n",
       "weight_prune_perc                                                               \n",
       "0.0                                19         0.396  0.960  0.912  0.090    60  \n",
       "0.1                                21         0.573  0.960  0.919  0.061    62  \n",
       "0.2                                23         0.836  0.960  0.936  0.022    56  \n",
       "0.3                                20         0.452  0.961  0.925  0.073    55  \n",
       "0.4                                22         0.392  0.963  0.918  0.093    56  \n",
       "0.5                                19         0.292  0.963  0.921  0.091    54  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['weight_prune_perc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>22</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.007</td>\n",
       "      <td>21</td>\n",
       "      <td>0.931</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.007</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>20</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.011</td>\n",
       "      <td>19</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.946</td>\n",
       "      <td>0.013</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>22</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.008</td>\n",
       "      <td>19</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.011</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>21</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.009</td>\n",
       "      <td>18</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.010</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>22</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.009</td>\n",
       "      <td>22</td>\n",
       "      <td>0.921</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.010</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>22</td>\n",
       "      <td>0.946</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.008</td>\n",
       "      <td>18</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.010</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                       \\\n",
       "                         round_mean         min    max   mean    std   \n",
       "weight_prune_perc                                                      \n",
       "0.0                              22       0.947  0.973  0.964  0.007   \n",
       "0.1                              20       0.932  0.976  0.961  0.011   \n",
       "0.2                              22       0.938  0.976  0.964  0.008   \n",
       "0.3                              21       0.940  0.976  0.964  0.009   \n",
       "0.4                              22       0.943  0.978  0.964  0.009   \n",
       "0.5                              22       0.946  0.976  0.964  0.008   \n",
       "\n",
       "                  noise_acc_max_epoch noise_acc_max                      model  \n",
       "                           round_mean           min    max   mean    std count  \n",
       "weight_prune_perc                                                               \n",
       "0.0                                21         0.931  0.960  0.949  0.007    22  \n",
       "0.1                                19         0.908  0.960  0.946  0.013    23  \n",
       "0.2                                19         0.911  0.960  0.949  0.011    23  \n",
       "0.3                                18         0.920  0.961  0.949  0.010    23  \n",
       "0.4                                22         0.921  0.963  0.949  0.010    23  \n",
       "0.5                                18         0.925  0.963  0.949  0.010    23  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['weight_prune_perc'], low_sparsity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>23</td>\n",
       "      <td>0.402</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.170</td>\n",
       "      <td>19</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.843</td>\n",
       "      <td>0.161</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>27</td>\n",
       "      <td>0.585</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.110</td>\n",
       "      <td>24</td>\n",
       "      <td>0.573</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.105</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>25</td>\n",
       "      <td>0.862</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.029</td>\n",
       "      <td>24</td>\n",
       "      <td>0.836</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.027</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>24</td>\n",
       "      <td>0.458</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.894</td>\n",
       "      <td>0.148</td>\n",
       "      <td>21</td>\n",
       "      <td>0.452</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.863</td>\n",
       "      <td>0.140</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>22</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.864</td>\n",
       "      <td>0.189</td>\n",
       "      <td>23</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.834</td>\n",
       "      <td>0.178</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>21</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.870</td>\n",
       "      <td>0.217</td>\n",
       "      <td>21</td>\n",
       "      <td>0.292</td>\n",
       "      <td>0.931</td>\n",
       "      <td>0.833</td>\n",
       "      <td>0.205</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                       \\\n",
       "                         round_mean         min    max   mean    std   \n",
       "weight_prune_perc                                                      \n",
       "0.0                              23       0.402  0.962  0.874  0.170   \n",
       "0.1                              27       0.585  0.968  0.904  0.110   \n",
       "0.2                              25       0.862  0.969  0.943  0.029   \n",
       "0.3                              24       0.458  0.966  0.894  0.148   \n",
       "0.4                              22       0.398  0.967  0.864  0.189   \n",
       "0.5                              21       0.295  0.964  0.870  0.217   \n",
       "\n",
       "                  noise_acc_max_epoch noise_acc_max                      model  \n",
       "                           round_mean           min    max   mean    std count  \n",
       "weight_prune_perc                                                               \n",
       "0.0                                19         0.396  0.925  0.843  0.161    15  \n",
       "0.1                                24         0.573  0.941  0.874  0.105    15  \n",
       "0.2                                24         0.836  0.940  0.911  0.027    13  \n",
       "0.3                                21         0.452  0.937  0.863  0.140    12  \n",
       "0.4                                23         0.392  0.936  0.834  0.178    12  \n",
       "0.5                                21         0.292  0.931  0.833  0.205     9  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['weight_prune_perc'], high_sparsity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>19</td>\n",
       "      <td>0.895</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.021</td>\n",
       "      <td>18</td>\n",
       "      <td>0.853</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.025</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>20</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.025</td>\n",
       "      <td>21</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.032</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>27</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.011</td>\n",
       "      <td>26</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.011</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>22</td>\n",
       "      <td>0.912</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.956</td>\n",
       "      <td>0.015</td>\n",
       "      <td>22</td>\n",
       "      <td>0.884</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.018</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>22</td>\n",
       "      <td>0.902</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.021</td>\n",
       "      <td>21</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.931</td>\n",
       "      <td>0.024</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>22</td>\n",
       "      <td>0.901</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.021</td>\n",
       "      <td>19</td>\n",
       "      <td>0.871</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.929</td>\n",
       "      <td>0.025</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                       \\\n",
       "                         round_mean         min    max   mean    std   \n",
       "weight_prune_perc                                                      \n",
       "0.0                              19       0.895  0.967  0.948  0.021   \n",
       "0.1                              20       0.888  0.970  0.947  0.025   \n",
       "0.2                              27       0.932  0.974  0.959  0.011   \n",
       "0.3                              22       0.912  0.974  0.956  0.015   \n",
       "0.4                              22       0.902  0.973  0.953  0.021   \n",
       "0.5                              22       0.901  0.974  0.951  0.021   \n",
       "\n",
       "                  noise_acc_max_epoch noise_acc_max                      model  \n",
       "                           round_mean           min    max   mean    std count  \n",
       "weight_prune_perc                                                               \n",
       "0.0                                18         0.853  0.943  0.922  0.025    23  \n",
       "0.1                                21         0.851  0.948  0.922  0.032    24  \n",
       "0.2                                26         0.909  0.954  0.938  0.011    20  \n",
       "0.3                                22         0.884  0.953  0.935  0.018    20  \n",
       "0.4                                21         0.873  0.952  0.931  0.024    21  \n",
       "0.5                                19         0.871  0.950  0.929  0.025    22  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['weight_prune_perc'], avg_sparsity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* In low sparsity cases, results are the same for any amount of pruning. In average and high sparsity, there is a gaussian like curve, with the peak at around 0.2 (maybe extending to 0.3). \n",
    "* Results are consistent with what has been seen in previous experiments and in related papers.\n",
    "* Worth note that although results are better at 0.2, it also takes slightly longer to achieve better results compared to m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>noise_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">noise_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>15</td>\n",
       "      <td>0.895</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.032</td>\n",
       "      <td>15</td>\n",
       "      <td>0.853</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.047</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>10</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.030</td>\n",
       "      <td>10</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.903</td>\n",
       "      <td>0.042</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>22</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.011</td>\n",
       "      <td>21</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.016</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>13</td>\n",
       "      <td>0.912</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.025</td>\n",
       "      <td>16</td>\n",
       "      <td>0.884</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.926</td>\n",
       "      <td>0.031</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>15</td>\n",
       "      <td>0.902</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.025</td>\n",
       "      <td>16</td>\n",
       "      <td>0.876</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.926</td>\n",
       "      <td>0.031</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>22</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.955</td>\n",
       "      <td>0.009</td>\n",
       "      <td>21</td>\n",
       "      <td>0.929</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.011</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                       \\\n",
       "                         round_mean         min    max   mean    std   \n",
       "weight_prune_perc                                                      \n",
       "0.0                              15       0.895  0.971  0.939  0.032   \n",
       "0.1                              10       0.888  0.960  0.928  0.030   \n",
       "0.2                              22       0.938  0.966  0.953  0.011   \n",
       "0.3                              13       0.912  0.965  0.944  0.025   \n",
       "0.4                              15       0.902  0.964  0.944  0.025   \n",
       "0.5                              22       0.943  0.963  0.955  0.009   \n",
       "\n",
       "                  noise_acc_max_epoch noise_acc_max                      model  \n",
       "                           round_mean           min    max   mean    std count  \n",
       "weight_prune_perc                                                               \n",
       "0.0                                15         0.853  0.959  0.915  0.047     5  \n",
       "0.1                                10         0.851  0.951  0.903  0.042     7  \n",
       "0.2                                21         0.911  0.953  0.937  0.016     5  \n",
       "0.3                                16         0.884  0.953  0.926  0.031     4  \n",
       "0.4                                16         0.876  0.953  0.926  0.031     5  \n",
       "0.5                                21         0.929  0.950  0.940  0.011     5  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_hebbian = (df['hebbian_prune_perc'] == 0)\n",
    "agg(['weight_prune_perc'], no_hebbian)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Somewhat inconsistent result looking at cases where there is no hebbian learning, only pruning by magnitude. There is an anomaly at the last entry where 50% of the weights are pruned - results are similar to 20%.\n",
    "* Number of samples averaged from is a lot lower in this pivot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### What is the optimal combination of weight and magnitude pruning?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.939 ± 0.032</td>\n",
       "      <td>0.928 ± 0.030</td>\n",
       "      <td>0.953 ± 0.011</td>\n",
       "      <td>0.944 ± 0.025</td>\n",
       "      <td>0.944 ± 0.025</td>\n",
       "      <td>0.955 ± 0.009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>0.940 ± 0.022</td>\n",
       "      <td>0.944 ± 0.017</td>\n",
       "      <td>0.950 ± 0.012</td>\n",
       "      <td>0.948 ± 0.011</td>\n",
       "      <td>0.935 ± 0.025</td>\n",
       "      <td>0.940 ± 0.024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.905 ± 0.167</td>\n",
       "      <td>0.914 ± 0.110</td>\n",
       "      <td>0.951 ± 0.015</td>\n",
       "      <td>0.908 ± 0.150</td>\n",
       "      <td>0.951 ± 0.019</td>\n",
       "      <td>0.943 ± 0.022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>0.906 ± 0.124</td>\n",
       "      <td>0.942 ± 0.058</td>\n",
       "      <td>0.957 ± 0.009</td>\n",
       "      <td>0.957 ± 0.012</td>\n",
       "      <td>0.921 ± 0.127</td>\n",
       "      <td>0.955 ± 0.015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.960 ± 0.010</td>\n",
       "      <td>0.949 ± 0.054</td>\n",
       "      <td>0.966 ± 0.007</td>\n",
       "      <td>0.968 ± 0.004</td>\n",
       "      <td>0.918 ± 0.164</td>\n",
       "      <td>0.966 ± 0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.963 ± 0.009</td>\n",
       "      <td>0.967 ± 0.007</td>\n",
       "      <td>0.960 ± 0.032</td>\n",
       "      <td>0.952 ± 0.058</td>\n",
       "      <td>0.970 ± 0.004</td>\n",
       "      <td>0.908 ± 0.204</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "weight_prune_perc             0.0            0.1            0.2  \\\n",
       "hebbian_prune_perc                                                \n",
       "0.0                 0.939 ± 0.032  0.928 ± 0.030  0.953 ± 0.011   \n",
       "0.1                 0.940 ± 0.022  0.944 ± 0.017  0.950 ± 0.012   \n",
       "0.2                 0.905 ± 0.167  0.914 ± 0.110  0.951 ± 0.015   \n",
       "0.3                 0.906 ± 0.124  0.942 ± 0.058  0.957 ± 0.009   \n",
       "0.4                 0.960 ± 0.010  0.949 ± 0.054  0.966 ± 0.007   \n",
       "0.5                 0.963 ± 0.009  0.967 ± 0.007  0.960 ± 0.032   \n",
       "\n",
       "weight_prune_perc             0.3            0.4            0.5  \n",
       "hebbian_prune_perc                                               \n",
       "0.0                 0.944 ± 0.025  0.944 ± 0.025  0.955 ± 0.009  \n",
       "0.1                 0.948 ± 0.011  0.935 ± 0.025  0.940 ± 0.024  \n",
       "0.2                 0.908 ± 0.150  0.951 ± 0.019  0.943 ± 0.022  \n",
       "0.3                 0.957 ± 0.012  0.921 ± 0.127  0.955 ± 0.015  \n",
       "0.4                 0.968 ± 0.004  0.918 ± 0.164  0.966 ± 0.005  \n",
       "0.5                 0.952 ± 0.058  0.970 ± 0.004  0.908 ± 0.204  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.pivot_table(df, \n",
    "              index='hebbian_prune_perc',\n",
    "              columns='weight_prune_perc',\n",
    "              values='val_acc_max',\n",
    "              aggfunc=mean_and_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kwinners</th>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">False</th>\n",
       "      <th>0.0</th>\n",
       "      <td>0.971 ± nan</td>\n",
       "      <td>0.946 ± 0.020</td>\n",
       "      <td>0.966 ± nan</td>\n",
       "      <td>0.951 ± 0.016</td>\n",
       "      <td>0.954 ± 0.015</td>\n",
       "      <td>0.963 ± nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>0.957 ± 0.011</td>\n",
       "      <td>0.952 ± 0.008</td>\n",
       "      <td>0.959 ± 0.003</td>\n",
       "      <td>0.955 ± 0.011</td>\n",
       "      <td>0.955 ± 0.014</td>\n",
       "      <td>0.954 ± 0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.960 ± 0.004</td>\n",
       "      <td>0.948 ± 0.014</td>\n",
       "      <td>0.960 ± 0.003</td>\n",
       "      <td>0.963 ± 0.006</td>\n",
       "      <td>0.963 ± 0.009</td>\n",
       "      <td>0.960 ± 0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>0.963 ± 0.001</td>\n",
       "      <td>0.965 ± 0.001</td>\n",
       "      <td>0.963 ± 0.002</td>\n",
       "      <td>0.966 ± 0.001</td>\n",
       "      <td>0.964 ± 0.002</td>\n",
       "      <td>0.964 ± 0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.967 ± 0.002</td>\n",
       "      <td>0.968 ± 0.004</td>\n",
       "      <td>0.969 ± 0.004</td>\n",
       "      <td>0.970 ± 0.002</td>\n",
       "      <td>0.970 ± 0.004</td>\n",
       "      <td>0.969 ± 0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.970 ± 0.003</td>\n",
       "      <td>0.971 ± 0.005</td>\n",
       "      <td>0.971 ± 0.005</td>\n",
       "      <td>0.972 ± 0.003</td>\n",
       "      <td>0.974 ± 0.006</td>\n",
       "      <td>0.973 ± 0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">True</th>\n",
       "      <th>0.0</th>\n",
       "      <td>0.959 ± nan</td>\n",
       "      <td>0.959 ± nan</td>\n",
       "      <td>0.950 ± 0.017</td>\n",
       "      <td>0.965 ± nan</td>\n",
       "      <td>0.963 ± nan</td>\n",
       "      <td>0.956 ± 0.010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>0.956 ± 0.012</td>\n",
       "      <td>0.954 ± 0.008</td>\n",
       "      <td>0.955 ± 0.005</td>\n",
       "      <td>0.955 ± 0.009</td>\n",
       "      <td>0.955 ± 0.011</td>\n",
       "      <td>0.958 ± 0.016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.966 ± 0.006</td>\n",
       "      <td>0.962 ± 0.002</td>\n",
       "      <td>0.964 ± 0.007</td>\n",
       "      <td>0.962 ± 0.007</td>\n",
       "      <td>0.961 ± 0.003</td>\n",
       "      <td>0.962 ± 0.007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>0.964 ± 0.000</td>\n",
       "      <td>0.964 ± 0.002</td>\n",
       "      <td>0.964 ± 0.002</td>\n",
       "      <td>0.967 ± 0.002</td>\n",
       "      <td>0.964 ± 0.005</td>\n",
       "      <td>0.965 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.967 ± 0.000</td>\n",
       "      <td>0.970 ± 0.003</td>\n",
       "      <td>0.969 ± 0.004</td>\n",
       "      <td>0.971 ± 0.002</td>\n",
       "      <td>0.970 ± 0.003</td>\n",
       "      <td>0.969 ± 0.007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.972 ± 0.002</td>\n",
       "      <td>0.973 ± 0.004</td>\n",
       "      <td>0.973 ± 0.004</td>\n",
       "      <td>0.973 ± 0.004</td>\n",
       "      <td>0.973 ± 0.003</td>\n",
       "      <td>0.974 ± 0.003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "weight_prune_perc                      0.0            0.1            0.2  \\\n",
       "kwinners hebbian_prune_perc                                                \n",
       "False    0.0                   0.971 ± nan  0.946 ± 0.020    0.966 ± nan   \n",
       "         0.1                 0.957 ± 0.011  0.952 ± 0.008  0.959 ± 0.003   \n",
       "         0.2                 0.960 ± 0.004  0.948 ± 0.014  0.960 ± 0.003   \n",
       "         0.3                 0.963 ± 0.001  0.965 ± 0.001  0.963 ± 0.002   \n",
       "         0.4                 0.967 ± 0.002  0.968 ± 0.004  0.969 ± 0.004   \n",
       "         0.5                 0.970 ± 0.003  0.971 ± 0.005  0.971 ± 0.005   \n",
       "True     0.0                   0.959 ± nan    0.959 ± nan  0.950 ± 0.017   \n",
       "         0.1                 0.956 ± 0.012  0.954 ± 0.008  0.955 ± 0.005   \n",
       "         0.2                 0.966 ± 0.006  0.962 ± 0.002  0.964 ± 0.007   \n",
       "         0.3                 0.964 ± 0.000  0.964 ± 0.002  0.964 ± 0.002   \n",
       "         0.4                 0.967 ± 0.000  0.970 ± 0.003  0.969 ± 0.004   \n",
       "         0.5                 0.972 ± 0.002  0.973 ± 0.004  0.973 ± 0.004   \n",
       "\n",
       "weight_prune_perc                      0.3            0.4            0.5  \n",
       "kwinners hebbian_prune_perc                                               \n",
       "False    0.0                 0.951 ± 0.016  0.954 ± 0.015    0.963 ± nan  \n",
       "         0.1                 0.955 ± 0.011  0.955 ± 0.014  0.954 ± 0.008  \n",
       "         0.2                 0.963 ± 0.006  0.963 ± 0.009  0.960 ± 0.003  \n",
       "         0.3                 0.966 ± 0.001  0.964 ± 0.002  0.964 ± 0.000  \n",
       "         0.4                 0.970 ± 0.002  0.970 ± 0.004  0.969 ± 0.004  \n",
       "         0.5                 0.972 ± 0.003  0.974 ± 0.006  0.973 ± 0.004  \n",
       "True     0.0                   0.965 ± nan    0.963 ± nan  0.956 ± 0.010  \n",
       "         0.1                 0.955 ± 0.009  0.955 ± 0.011  0.958 ± 0.016  \n",
       "         0.2                 0.962 ± 0.007  0.961 ± 0.003  0.962 ± 0.007  \n",
       "         0.3                 0.967 ± 0.002  0.964 ± 0.005  0.965 ± 0.002  \n",
       "         0.4                 0.971 ± 0.002  0.970 ± 0.003  0.969 ± 0.007  \n",
       "         0.5                 0.973 ± 0.004  0.973 ± 0.003  0.974 ± 0.003  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.pivot_table(df[low_sparsity], \n",
    "              index=['kwinners','hebbian_prune_perc'],\n",
    "              columns='weight_prune_perc',\n",
    "              values='val_acc_max',\n",
    "              aggfunc=mean_and_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kwinners</th>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">False</th>\n",
       "      <th>0.0</th>\n",
       "      <td>0.925 ± 0.042</td>\n",
       "      <td>0.903 ± 0.000</td>\n",
       "      <td>0.950 ± nan</td>\n",
       "      <td>0.912 ± nan</td>\n",
       "      <td>0.902 ± nan</td>\n",
       "      <td>0.956 ± nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>0.924 ± 0.026</td>\n",
       "      <td>0.943 ± 0.011</td>\n",
       "      <td>0.931 ± nan</td>\n",
       "      <td>0.936 ± nan</td>\n",
       "      <td>0.915 ± 0.016</td>\n",
       "      <td>0.911 ± 0.014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.955 ± 0.004</td>\n",
       "      <td>0.930 ± 0.034</td>\n",
       "      <td>0.955 ± 0.003</td>\n",
       "      <td>0.950 ± 0.005</td>\n",
       "      <td>0.953 ± 0.007</td>\n",
       "      <td>0.950 ± 0.007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>0.954 ± 0.011</td>\n",
       "      <td>0.958 ± 0.005</td>\n",
       "      <td>0.959 ± 0.003</td>\n",
       "      <td>0.960 ± 0.002</td>\n",
       "      <td>0.961 ± 0.002</td>\n",
       "      <td>0.961 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.963 ± 0.003</td>\n",
       "      <td>0.965 ± 0.006</td>\n",
       "      <td>0.969 ± 0.005</td>\n",
       "      <td>0.968 ± 0.006</td>\n",
       "      <td>0.968 ± 0.002</td>\n",
       "      <td>0.967 ± 0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.965 ± 0.002</td>\n",
       "      <td>0.968 ± 0.003</td>\n",
       "      <td>0.970 ± 0.005</td>\n",
       "      <td>0.969 ± 0.004</td>\n",
       "      <td>0.970 ± 0.003</td>\n",
       "      <td>0.969 ± 0.007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">True</th>\n",
       "      <th>0.0</th>\n",
       "      <td>0.915 ± nan</td>\n",
       "      <td>0.919 ± 0.044</td>\n",
       "      <td>0.950 ± nan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.946 ± nan</td>\n",
       "      <td>0.943 ± nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>0.923 ± 0.018</td>\n",
       "      <td>0.926 ± 0.027</td>\n",
       "      <td>0.942 ± nan</td>\n",
       "      <td>0.940 ± 0.004</td>\n",
       "      <td>0.929 ± nan</td>\n",
       "      <td>0.937 ± 0.029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.949 ± 0.008</td>\n",
       "      <td>0.954 ± 0.001</td>\n",
       "      <td>0.950 ± 0.001</td>\n",
       "      <td>0.956 ± 0.006</td>\n",
       "      <td>0.957 ± 0.007</td>\n",
       "      <td>0.928 ± 0.027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>0.954 ± 0.009</td>\n",
       "      <td>0.961 ± 0.004</td>\n",
       "      <td>0.959 ± 0.001</td>\n",
       "      <td>0.962 ± 0.004</td>\n",
       "      <td>0.960 ± 0.004</td>\n",
       "      <td>0.958 ± 0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.964 ± 0.005</td>\n",
       "      <td>0.966 ± 0.006</td>\n",
       "      <td>0.967 ± 0.007</td>\n",
       "      <td>0.964 ± 0.006</td>\n",
       "      <td>0.964 ± 0.008</td>\n",
       "      <td>0.966 ± 0.007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.965 ± 0.004</td>\n",
       "      <td>0.968 ± 0.003</td>\n",
       "      <td>0.970 ± 0.004</td>\n",
       "      <td>0.970 ± 0.005</td>\n",
       "      <td>0.970 ± 0.004</td>\n",
       "      <td>0.968 ± 0.006</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "weight_prune_perc                      0.0            0.1            0.2  \\\n",
       "kwinners hebbian_prune_perc                                                \n",
       "False    0.0                 0.925 ± 0.042  0.903 ± 0.000    0.950 ± nan   \n",
       "         0.1                 0.924 ± 0.026  0.943 ± 0.011    0.931 ± nan   \n",
       "         0.2                 0.955 ± 0.004  0.930 ± 0.034  0.955 ± 0.003   \n",
       "         0.3                 0.954 ± 0.011  0.958 ± 0.005  0.959 ± 0.003   \n",
       "         0.4                 0.963 ± 0.003  0.965 ± 0.006  0.969 ± 0.005   \n",
       "         0.5                 0.965 ± 0.002  0.968 ± 0.003  0.970 ± 0.005   \n",
       "True     0.0                   0.915 ± nan  0.919 ± 0.044    0.950 ± nan   \n",
       "         0.1                 0.923 ± 0.018  0.926 ± 0.027    0.942 ± nan   \n",
       "         0.2                 0.949 ± 0.008  0.954 ± 0.001  0.950 ± 0.001   \n",
       "         0.3                 0.954 ± 0.009  0.961 ± 0.004  0.959 ± 0.001   \n",
       "         0.4                 0.964 ± 0.005  0.966 ± 0.006  0.967 ± 0.007   \n",
       "         0.5                 0.965 ± 0.004  0.968 ± 0.003  0.970 ± 0.004   \n",
       "\n",
       "weight_prune_perc                      0.3            0.4            0.5  \n",
       "kwinners hebbian_prune_perc                                               \n",
       "False    0.0                   0.912 ± nan    0.902 ± nan    0.956 ± nan  \n",
       "         0.1                   0.936 ± nan  0.915 ± 0.016  0.911 ± 0.014  \n",
       "         0.2                 0.950 ± 0.005  0.953 ± 0.007  0.950 ± 0.007  \n",
       "         0.3                 0.960 ± 0.002  0.961 ± 0.002  0.961 ± 0.001  \n",
       "         0.4                 0.968 ± 0.006  0.968 ± 0.002  0.967 ± 0.004  \n",
       "         0.5                 0.969 ± 0.004  0.970 ± 0.003  0.969 ± 0.007  \n",
       "True     0.0                           NaN    0.946 ± nan    0.943 ± nan  \n",
       "         0.1                 0.940 ± 0.004    0.929 ± nan  0.937 ± 0.029  \n",
       "         0.2                 0.956 ± 0.006  0.957 ± 0.007  0.928 ± 0.027  \n",
       "         0.3                 0.962 ± 0.004  0.960 ± 0.004  0.958 ± 0.003  \n",
       "         0.4                 0.964 ± 0.006  0.964 ± 0.008  0.966 ± 0.007  \n",
       "         0.5                 0.970 ± 0.005  0.970 ± 0.004  0.968 ± 0.006  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.pivot_table(df[avg_sparsity], \n",
    "              index=['kwinners','hebbian_prune_perc'],\n",
    "              columns='weight_prune_perc',\n",
    "              values='val_acc_max',\n",
    "              aggfunc=mean_and_std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* There is a more clear trend in the low sparsity case. Results from high sparsity are inconclusive, with several runs failing to \"converge\"\n",
    "* Weight pruning alone improves the model by up to 0.7% from 10% pruning to 50% magnitude pruning\n",
    "* Hebbian pruning alone improves the model by 1.5%\n",
    "* Both combined can increase from 1.5% seem in hebbian only to 1.8% improvement. \n",
    "* Comparisons above are from 0.1 to 0.5 pruning. There is a question left of why no pruning at both sides - the (0,0) point - it is an anomaly to the trend shown in the pivot."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.921 ± 0.030</td>\n",
       "      <td>0.911 ± 0.027</td>\n",
       "      <td>0.950 ± 0.000</td>\n",
       "      <td>0.912 ± nan</td>\n",
       "      <td>0.924 ± 0.031</td>\n",
       "      <td>0.950 ± 0.009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>0.923 ± 0.018</td>\n",
       "      <td>0.934 ± 0.019</td>\n",
       "      <td>0.937 ± 0.007</td>\n",
       "      <td>0.939 ± 0.003</td>\n",
       "      <td>0.919 ± 0.014</td>\n",
       "      <td>0.924 ± 0.024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.952 ± 0.006</td>\n",
       "      <td>0.942 ± 0.024</td>\n",
       "      <td>0.953 ± 0.003</td>\n",
       "      <td>0.953 ± 0.006</td>\n",
       "      <td>0.955 ± 0.006</td>\n",
       "      <td>0.939 ± 0.020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>0.954 ± 0.008</td>\n",
       "      <td>0.960 ± 0.004</td>\n",
       "      <td>0.959 ± 0.002</td>\n",
       "      <td>0.961 ± 0.003</td>\n",
       "      <td>0.961 ± 0.003</td>\n",
       "      <td>0.959 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.964 ± 0.004</td>\n",
       "      <td>0.965 ± 0.005</td>\n",
       "      <td>0.968 ± 0.005</td>\n",
       "      <td>0.966 ± 0.005</td>\n",
       "      <td>0.966 ± 0.005</td>\n",
       "      <td>0.967 ± 0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.965 ± 0.003</td>\n",
       "      <td>0.968 ± 0.002</td>\n",
       "      <td>0.970 ± 0.004</td>\n",
       "      <td>0.970 ± 0.004</td>\n",
       "      <td>0.970 ± 0.003</td>\n",
       "      <td>0.969 ± 0.005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "weight_prune_perc             0.0            0.1            0.2  \\\n",
       "hebbian_prune_perc                                                \n",
       "0.0                 0.921 ± 0.030  0.911 ± 0.027  0.950 ± 0.000   \n",
       "0.1                 0.923 ± 0.018  0.934 ± 0.019  0.937 ± 0.007   \n",
       "0.2                 0.952 ± 0.006  0.942 ± 0.024  0.953 ± 0.003   \n",
       "0.3                 0.954 ± 0.008  0.960 ± 0.004  0.959 ± 0.002   \n",
       "0.4                 0.964 ± 0.004  0.965 ± 0.005  0.968 ± 0.005   \n",
       "0.5                 0.965 ± 0.003  0.968 ± 0.002  0.970 ± 0.004   \n",
       "\n",
       "weight_prune_perc             0.3            0.4            0.5  \n",
       "hebbian_prune_perc                                               \n",
       "0.0                   0.912 ± nan  0.924 ± 0.031  0.950 ± 0.009  \n",
       "0.1                 0.939 ± 0.003  0.919 ± 0.014  0.924 ± 0.024  \n",
       "0.2                 0.953 ± 0.006  0.955 ± 0.006  0.939 ± 0.020  \n",
       "0.3                 0.961 ± 0.003  0.961 ± 0.003  0.959 ± 0.002  \n",
       "0.4                 0.966 ± 0.005  0.966 ± 0.005  0.967 ± 0.004  \n",
       "0.5                 0.970 ± 0.004  0.970 ± 0.003  0.969 ± 0.005  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.pivot_table(df[avg_sparsity], \n",
    "              index='hebbian_prune_perc',\n",
    "              columns='weight_prune_perc',\n",
    "              values='val_acc_max',\n",
    "              aggfunc=mean_and_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.900 ± nan</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.764 ± 0.313</td>\n",
       "      <td>0.821 ± 0.205</td>\n",
       "      <td>0.932 ± 0.017</td>\n",
       "      <td>0.776 ± 0.276</td>\n",
       "      <td>0.919 ± 0.020</td>\n",
       "      <td>0.916 ± 0.011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>0.802 ± 0.185</td>\n",
       "      <td>0.901 ± 0.095</td>\n",
       "      <td>0.945 ± 0.007</td>\n",
       "      <td>0.940 ± 0.007</td>\n",
       "      <td>0.811 ± 0.237</td>\n",
       "      <td>0.927 ± 0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.948 ± 0.010</td>\n",
       "      <td>0.912 ± 0.090</td>\n",
       "      <td>0.959 ± 0.008</td>\n",
       "      <td>0.965 ± 0.001</td>\n",
       "      <td>0.817 ± 0.280</td>\n",
       "      <td>0.960 ± 0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.953 ± 0.009</td>\n",
       "      <td>0.960 ± 0.009</td>\n",
       "      <td>0.937 ± 0.051</td>\n",
       "      <td>0.912 ± 0.096</td>\n",
       "      <td>0.965 ± 0.002</td>\n",
       "      <td>0.740 ± 0.386</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "weight_prune_perc             0.0            0.1            0.2  \\\n",
       "hebbian_prune_perc                                                \n",
       "0.1                           NaN            NaN            NaN   \n",
       "0.2                 0.764 ± 0.313  0.821 ± 0.205  0.932 ± 0.017   \n",
       "0.3                 0.802 ± 0.185  0.901 ± 0.095  0.945 ± 0.007   \n",
       "0.4                 0.948 ± 0.010  0.912 ± 0.090  0.959 ± 0.008   \n",
       "0.5                 0.953 ± 0.009  0.960 ± 0.009  0.937 ± 0.051   \n",
       "\n",
       "weight_prune_perc             0.3            0.4            0.5  \n",
       "hebbian_prune_perc                                               \n",
       "0.1                           NaN    0.900 ± nan            NaN  \n",
       "0.2                 0.776 ± 0.276  0.919 ± 0.020  0.916 ± 0.011  \n",
       "0.3                 0.940 ± 0.007  0.811 ± 0.237  0.927 ± 0.008  \n",
       "0.4                 0.965 ± 0.001  0.817 ± 0.280  0.960 ± 0.003  \n",
       "0.5                 0.912 ± 0.096  0.965 ± 0.002  0.740 ± 0.386  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.pivot_table(df[high_sparsity], \n",
    "              index='hebbian_prune_perc',\n",
    "              columns='weight_prune_perc',\n",
    "              values='val_acc_max',\n",
    "              aggfunc=mean_and_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
